import pandas as pd
import re
path_1 = 'Auto_label_data/fasttext_auto_label_0719.csv'
path_2 = 'Auto_label_data/result/bert_result_0719.csv'


def output_txt(data, name):
    with open(f'Auto_label_data/result/{name}.txt', 'w+', encoding='utf-8') as f:
        for line in data:
            f.write(line + '\n')
    print("Done!")


def diff_extract(df1, df2):
    differences = []  # 存储不同的行
    labels_1, labels_2 = list(df1['label']), list(df2['label'])
    diff_index = []
    others, others_idx = [], []
    fasttext_result, bert_result = [], []
    same_text, same_label = [], []
    diff_contain_title_push, other_contain_title_push = [], []
    for i in range(len(labels_1)):
        if labels_1[i] != labels_2[i]:
            diff_contain_title_push.append(df1.loc[i]['data'])
            differences.append(df2.loc[i]['text'])
            diff_index.append(i)
            fasttext_result.append(labels_1[i])
            bert_result.append(labels_2[i])

        elif labels_1[i] == '其他-其他-其他' and labels_2[i] == '其他-其他-其他':
            other_contain_title_push.append(df1.loc[i]['data'])
            others.append(df2.loc[i]['text'])
            others_idx.append(i)
    diff_df = pd.DataFrame(
        {"diff_index": diff_index, "diff_content": differences, "fasttext": fasttext_result, "bert": bert_result})
    # diff_df.to_csv('Auto_label_data/result/diff_result_0719.csv.txt')
    # other_df = pd.DataFrame(
    #     {"diff_index": others_idx, "other_content": others})
    # other_df.to_csv('Auto_label_data/result/other_result.csv.txt')
    # output_txt(others, 'others')
    diff_index.extend(others_idx)
    same_df = df2.drop(diff_index, axis=0)
    same_push = list(df1.drop(diff_index, axis=0)['data'])
    # same_df.to_csv('Auto_label_data/result/same_result_0719.csv.txt')
    # output_txt(differences, 'difference')

    """以下方法仅用于还原app——name/及区分title， 用于一些特殊数据处理"""
    print(len(differences) + len(others))
    contain_title_push = diff_contain_title_push + other_contain_title_push
    final = differences + others
    # extract_detail_info
    all_commented_data = pd.read_csv(r'D:\pythonproject\Bert_Classification\0830_data\all.csv')
    all_commented_data.drop(columns=['id', 'Comments'], inplace=True)
    all_commented_data['text'] = all_commented_data['text'].apply(lambda x: x.replace('"""', ''))
    all_commented_title = []
    for push in contain_title_push:
        # 无监督数据
        pattern_2 = r"(title__)([^_]+)"
        matches = re.findall(pattern_2, push)[0]
        all_commented_title.append(matches[-1])
    app_name = []
    push_info = []

    for diff in final:
        # to get app_name;
        app_name.append(diff.split('-')[0])
        push_info.append(diff.split('-')[-1])

    all_commented_data['app_name'] = app_name
    all_commented_data['text'] = push_info
    all_commented_data['title'] = all_commented_title

    # 處理 same_df 還原app_name
    same_push = list(df1.drop(diff_index, axis=0)['data'])
    same_push_title = []
    for push in same_push:
        pattern_2 = r"(title__)([^_]+)"
        matches = re.findall(pattern_2, push)[0]
        same_push_title.append(matches[-1])

    new_same_app_name, new_same_text = [], []
    for text in list(same_df['text']):
        new_same_app_name.append(text.split('-')[0])
        new_same_text.append(text.split('-')[1])

    same_df['title'] = same_push_title
    same_df['text'] = new_same_text
    same_df['app_name'] = new_same_app_name
    same_df['label'] = same_df['label'].apply(lambda x: x.replace('公信-', ''))
    same_df['label'] = same_df['label'].apply(lambda x: x.replace('私信-', ''))
    same_df.drop(columns=['Unnamed: 0'], inplace=True)
    all_commented_data.to_csv(r'D:\pythonproject\Bert_Classification\0830_data\new_all.csv', index=False)
    final_df = pd.concat([same_df, all_commented_data])
    final_df.drop_duplicates(subset=['text'], keep='first', inplace=True)
    final_df.to_csv(r'D:\pythonproject\Bert_Classification\0830_data\final_new_all.csv', index=False)


def merging_operation():
    df1 = pd.read_csv(r'D:\pythonproject\Bert_Classification\0830_data\final_new_all.csv')
    df2 = pd.read_csv(r'D:\pythonproject\Bert_Classification\0830_data\drop_duplicate_version.csv')
    df2.drop(columns=['Unnamed: 0', 'Unnamed: 0.1'], inplace=True)
    df1.rename(columns={"text": "content"}, inplace=True)
    new_df = pd.merge(df1, df2, left_on=['app_name', 'content', 'title'], right_on=['app_name', 'content', 'title'], how='outer')
    print(new_df)
    # new_df = pd.concat([df1, df2])
    new_df.to_csv(r'D:\pythonproject\Bert_Classification\0830_data\final_drop_duplicate.csv', index=False)


def same_data_filter():
    df = pd.read_csv(r'D:\pythonproject\Bert_Classification\0830_data\final_drop_duplicate.csv')
    push = df['content']
    # assert len(push) == len(set(df['content']))
    from collections import Counter
    counter = Counter(push)
    print(counter.most_common()[:10])


if __name__ == '__main__':
    # data_1 = pd.read_csv(path_1)
    # data_2 = pd.read_csv(path_2)
    # diff_extract(data_1, data_2)
    # merging_operation()
    same_data_filter()